{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "id": "1843a210-cdd5-4396-beab-0e68cd0d493d",
   "metadata": {},
   "outputs": [],
   "source": [
    "from spider import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 2,
   "id": "965666d0-7b10-45ee-baee-a3889a6d7df3",
   "metadata": {},
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "page 1: len(links) = 1\n",
      "page 1: len(links) = 1\n"
     ]
    },
    {
     "ename": "StaleElementReferenceException",
     "evalue": "Message: stale element reference: stale element not found in the current frame\n  (Session info: MicrosoftEdge=132.0.2957.140); For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors#stale-element-reference-exception\nStacktrace:\n#0 0x59c9e40df70e <unknown>\n#1 0x59c9e3b7fab6 <unknown>\n#2 0x59c9e3b85836 <unknown>\n#3 0x59c9e3b87b7a <unknown>\n#4 0x59c9e3c1139e <unknown>\n#5 0x59c9e3bf2722 <unknown>\n#6 0x59c9e3c106dd <unknown>\n#7 0x59c9e3bf24c3 <unknown>\n#8 0x59c9e3bc0efe <unknown>\n#9 0x59c9e3bc1c5e <unknown>\n#10 0x59c9e40b35c1 <unknown>\n#11 0x59c9e40b67be <unknown>\n#12 0x59c9e40b623b <unknown>\n#13 0x59c9e40b6be5 <unknown>\n#14 0x59c9e40a4d82 <unknown>\n#15 0x59c9e40b6f8d <unknown>\n#16 0x59c9e408df65 <unknown>\n#17 0x59c9e40d0708 <unknown>\n#18 0x59c9e40d091f <unknown>\n#19 0x59c9e40deb1c <unknown>\n#20 0x7c7bc329a39d <unknown>\n#21 0x7c7bc331f49c <unknown>\n",
     "output_type": "error",
     "traceback": [
      "\u001b[0;31m---------------------------------------------------------------------------\u001b[0m",
      "\u001b[0;31mStaleElementReferenceException\u001b[0m            Traceback (most recent call last)",
      "File \u001b[0;32m~/.venv/default/lib/python3.12/site-packages/selenium/webdriver/remote/webelement.py:173\u001b[0m, in \u001b[0;36mWebElement.get_property\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m    172\u001b[0m \u001b[38;5;28;01mtry\u001b[39;00m:\n\u001b[0;32m--> 173\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_execute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mCommand\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mGET_ELEMENT_PROPERTY\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mname\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m    174\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m WebDriverException:\n\u001b[1;32m    175\u001b[0m     \u001b[38;5;66;03m# if we hit an end point that doesn't understand getElementProperty lets fake it\u001b[39;00m\n",
      "File \u001b[0;32m~/.venv/default/lib/python3.12/site-packages/selenium/webdriver/remote/webelement.py:570\u001b[0m, in \u001b[0;36mWebElement._execute\u001b[0;34m(self, command, params)\u001b[0m\n\u001b[1;32m    569\u001b[0m params[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mid\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_id\n\u001b[0;32m--> 570\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43m_parent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcommand\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mparams\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.venv/default/lib/python3.12/site-packages/selenium/webdriver/remote/webdriver.py:429\u001b[0m, in \u001b[0;36mWebDriver.execute\u001b[0;34m(self, driver_command, params)\u001b[0m\n\u001b[1;32m    428\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m response:\n\u001b[0;32m--> 429\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43merror_handler\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcheck_response\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    430\u001b[0m     response[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_unwrap_value(response\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m))\n",
      "File \u001b[0;32m~/.venv/default/lib/python3.12/site-packages/selenium/webdriver/remote/errorhandler.py:232\u001b[0m, in \u001b[0;36mErrorHandler.check_response\u001b[0;34m(self, response)\u001b[0m\n\u001b[1;32m    231\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m exception_class(message, screen, stacktrace, alert_text)  \u001b[38;5;66;03m# type: ignore[call-arg]  # mypy is not smart enough here\u001b[39;00m\n\u001b[0;32m--> 232\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exception_class(message, screen, stacktrace)\n",
      "\u001b[0;31mStaleElementReferenceException\u001b[0m: Message: stale element reference: stale element not found in the current frame\n  (Session info: MicrosoftEdge=132.0.2957.140); For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors#stale-element-reference-exception\nStacktrace:\n#0 0x59c9e40df70e <unknown>\n#1 0x59c9e3b7fab6 <unknown>\n#2 0x59c9e3b85836 <unknown>\n#3 0x59c9e3b87859 <unknown>\n#4 0x59c9e3b87914 <unknown>\n#5 0x59c9e3bc62be <unknown>\n#6 0x59c9e3bf2722 <unknown>\n#7 0x59c9e3bc2b51 <unknown>\n#8 0x59c9e3bf28ee <unknown>\n#9 0x59c9e3c106dd <unknown>\n#10 0x59c9e3bf24c3 <unknown>\n#11 0x59c9e3bc0efe <unknown>\n#12 0x59c9e3bc1c5e <unknown>\n#13 0x59c9e40b35c1 <unknown>\n#14 0x59c9e40b67be <unknown>\n#15 0x59c9e40b623b <unknown>\n#16 0x59c9e40b6be5 <unknown>\n#17 0x59c9e40a4d82 <unknown>\n#18 0x59c9e40b6f8d <unknown>\n#19 0x59c9e408df65 <unknown>\n#20 0x59c9e40d0708 <unknown>\n#21 0x59c9e40d091f <unknown>\n#22 0x59c9e40deb1c <unknown>\n#23 0x7c7bc329a39d <unknown>\n#24 0x7c7bc331f49c <unknown>\n",
      "\nDuring handling of the above exception, another exception occurred:\n",
      "\u001b[0;31mStaleElementReferenceException\u001b[0m            Traceback (most recent call last)",
      "Cell \u001b[0;32mIn[2], line 1\u001b[0m\n\u001b[0;32m----> 1\u001b[0m repos \u001b[38;5;241m=\u001b[39m \u001b[43mmain\u001b[49m\u001b[43m(\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/Projects/PyLearning/notebook/spider.py:99\u001b[0m, in \u001b[0;36mmain\u001b[0;34m(url, driver, selector, parent, timeout, **kwargs)\u001b[0m\n\u001b[1;32m     97\u001b[0m \u001b[38;5;28;01mwhile\u001b[39;00m \u001b[38;5;28mlen\u001b[39m(links) \u001b[38;5;241m>\u001b[39m \u001b[38;5;241m0\u001b[39m:\n\u001b[1;32m     98\u001b[0m     active_page \u001b[38;5;241m=\u001b[39m find_page(driver, \u001b[38;5;124m'\u001b[39m\u001b[38;5;124m[@class=\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mitem active\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124m]\u001b[39m\u001b[38;5;124m'\u001b[39m, parent)\n\u001b[0;32m---> 99\u001b[0m     repos \u001b[38;5;241m=\u001b[39m repos \u001b[38;5;241m+\u001b[39m \u001b[43mget_repositories\u001b[49m\u001b[43m(\u001b[49m\u001b[43mdriver\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    100\u001b[0m     \u001b[38;5;28mprint\u001b[39m(\u001b[38;5;124mf\u001b[39m\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mpage \u001b[39m\u001b[38;5;132;01m{\u001b[39;00mactive_page\u001b[38;5;241m.\u001b[39mtext\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m: len(links) = \u001b[39m\u001b[38;5;132;01m{\u001b[39;00m\u001b[38;5;28mlen\u001b[39m(links)\u001b[38;5;132;01m}\u001b[39;00m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    101\u001b[0m     links[\u001b[38;5;241m0\u001b[39m]\u001b[38;5;241m.\u001b[39mclick()\n",
      "File \u001b[0;32m~/Projects/PyLearning/notebook/spider.py:57\u001b[0m, in \u001b[0;36mget_repositories\u001b[0;34m(driver, selector, parent, timeout, **kwargs)\u001b[0m\n\u001b[1;32m     55\u001b[0m xpath \u001b[38;5;241m=\u001b[39m parent \u001b[38;5;241m+\u001b[39m selector\n\u001b[1;32m     56\u001b[0m driver\u001b[38;5;241m.\u001b[39mimplicitly_wait(timeout)\n\u001b[0;32m---> 57\u001b[0m links \u001b[38;5;241m=\u001b[39m [\u001b[43me\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mget_property\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[38;5;124;43mhref\u001b[39;49m\u001b[38;5;124;43m'\u001b[39;49m\u001b[43m)\u001b[49m \u001b[38;5;28;01mfor\u001b[39;00m e \u001b[38;5;129;01min\u001b[39;00m driver\u001b[38;5;241m.\u001b[39mfind_elements(By\u001b[38;5;241m.\u001b[39mXPATH, xpath)]\n\u001b[1;32m     58\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m links\n",
      "File \u001b[0;32m~/.venv/default/lib/python3.12/site-packages/selenium/webdriver/remote/webelement.py:176\u001b[0m, in \u001b[0;36mWebElement.get_property\u001b[0;34m(self, name)\u001b[0m\n\u001b[1;32m    173\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_execute(Command\u001b[38;5;241m.\u001b[39mGET_ELEMENT_PROPERTY, {\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mname\u001b[39m\u001b[38;5;124m\"\u001b[39m: name})[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n\u001b[1;32m    174\u001b[0m \u001b[38;5;28;01mexcept\u001b[39;00m WebDriverException:\n\u001b[1;32m    175\u001b[0m     \u001b[38;5;66;03m# if we hit an end point that doesn't understand getElementProperty lets fake it\u001b[39;00m\n\u001b[0;32m--> 176\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mparent\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute_script\u001b[49m\u001b[43m(\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mreturn arguments[0][arguments[1]]\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;28;43mself\u001b[39;49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43mname\u001b[49m\u001b[43m)\u001b[49m\n",
      "File \u001b[0;32m~/.venv/default/lib/python3.12/site-packages/selenium/webdriver/remote/webdriver.py:528\u001b[0m, in \u001b[0;36mWebDriver.execute_script\u001b[0;34m(self, script, *args)\u001b[0m\n\u001b[1;32m    525\u001b[0m converted_args \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mlist\u001b[39m(args)\n\u001b[1;32m    526\u001b[0m command \u001b[38;5;241m=\u001b[39m Command\u001b[38;5;241m.\u001b[39mW3C_EXECUTE_SCRIPT\n\u001b[0;32m--> 528\u001b[0m \u001b[38;5;28;01mreturn\u001b[39;00m \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mexecute\u001b[49m\u001b[43m(\u001b[49m\u001b[43mcommand\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[43m{\u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43mscript\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mscript\u001b[49m\u001b[43m,\u001b[49m\u001b[43m \u001b[49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[38;5;124;43margs\u001b[39;49m\u001b[38;5;124;43m\"\u001b[39;49m\u001b[43m:\u001b[49m\u001b[43m \u001b[49m\u001b[43mconverted_args\u001b[49m\u001b[43m}\u001b[49m\u001b[43m)\u001b[49m[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m]\n",
      "File \u001b[0;32m~/.venv/default/lib/python3.12/site-packages/selenium/webdriver/remote/webdriver.py:429\u001b[0m, in \u001b[0;36mWebDriver.execute\u001b[0;34m(self, driver_command, params)\u001b[0m\n\u001b[1;32m    427\u001b[0m response \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39mcommand_executor\u001b[38;5;241m.\u001b[39mexecute(driver_command, params)\n\u001b[1;32m    428\u001b[0m \u001b[38;5;28;01mif\u001b[39;00m response:\n\u001b[0;32m--> 429\u001b[0m     \u001b[38;5;28;43mself\u001b[39;49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43merror_handler\u001b[49m\u001b[38;5;241;43m.\u001b[39;49m\u001b[43mcheck_response\u001b[49m\u001b[43m(\u001b[49m\u001b[43mresponse\u001b[49m\u001b[43m)\u001b[49m\n\u001b[1;32m    430\u001b[0m     response[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m] \u001b[38;5;241m=\u001b[39m \u001b[38;5;28mself\u001b[39m\u001b[38;5;241m.\u001b[39m_unwrap_value(response\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mvalue\u001b[39m\u001b[38;5;124m\"\u001b[39m, \u001b[38;5;28;01mNone\u001b[39;00m))\n\u001b[1;32m    431\u001b[0m     \u001b[38;5;28;01mreturn\u001b[39;00m response\n",
      "File \u001b[0;32m~/.venv/default/lib/python3.12/site-packages/selenium/webdriver/remote/errorhandler.py:232\u001b[0m, in \u001b[0;36mErrorHandler.check_response\u001b[0;34m(self, response)\u001b[0m\n\u001b[1;32m    230\u001b[0m         alert_text \u001b[38;5;241m=\u001b[39m value[\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124malert\u001b[39m\u001b[38;5;124m\"\u001b[39m]\u001b[38;5;241m.\u001b[39mget(\u001b[38;5;124m\"\u001b[39m\u001b[38;5;124mtext\u001b[39m\u001b[38;5;124m\"\u001b[39m)\n\u001b[1;32m    231\u001b[0m     \u001b[38;5;28;01mraise\u001b[39;00m exception_class(message, screen, stacktrace, alert_text)  \u001b[38;5;66;03m# type: ignore[call-arg]  # mypy is not smart enough here\u001b[39;00m\n\u001b[0;32m--> 232\u001b[0m \u001b[38;5;28;01mraise\u001b[39;00m exception_class(message, screen, stacktrace)\n",
      "\u001b[0;31mStaleElementReferenceException\u001b[0m: Message: stale element reference: stale element not found in the current frame\n  (Session info: MicrosoftEdge=132.0.2957.140); For documentation on this error, please visit: https://www.selenium.dev/documentation/webdriver/troubleshooting/errors#stale-element-reference-exception\nStacktrace:\n#0 0x59c9e40df70e <unknown>\n#1 0x59c9e3b7fab6 <unknown>\n#2 0x59c9e3b85836 <unknown>\n#3 0x59c9e3b87b7a <unknown>\n#4 0x59c9e3c1139e <unknown>\n#5 0x59c9e3bf2722 <unknown>\n#6 0x59c9e3c106dd <unknown>\n#7 0x59c9e3bf24c3 <unknown>\n#8 0x59c9e3bc0efe <unknown>\n#9 0x59c9e3bc1c5e <unknown>\n#10 0x59c9e40b35c1 <unknown>\n#11 0x59c9e40b67be <unknown>\n#12 0x59c9e40b623b <unknown>\n#13 0x59c9e40b6be5 <unknown>\n#14 0x59c9e40a4d82 <unknown>\n#15 0x59c9e40b6f8d <unknown>\n#16 0x59c9e408df65 <unknown>\n#17 0x59c9e40d0708 <unknown>\n#18 0x59c9e40d091f <unknown>\n#19 0x59c9e40deb1c <unknown>\n#20 0x7c7bc329a39d <unknown>\n#21 0x7c7bc331f49c <unknown>\n"
     ]
    }
   ],
   "source": [
    "repos = main()"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "id": "10b6e911-8f45-40e3-98e0-5eccbfe78915",
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3 (ipykernel)",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.12.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 5
}
